//////////////////////////////////////////////////////////////////
// (c) Copyright 2008- by Jeongnim Kim
//////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////
//   National Center for Supercomputing Applications &
//   Materials Computation Center
//   University of Illinois, Urbana-Champaign
//   Urbana, IL 61801
//   e-mail: jnkim@ncsa.uiuc.edu
//
// Supported by 
//   National Center for Supercomputing Applications, UIUC
//   Materials Computation Center, UIUC
//////////////////////////////////////////////////////////////////
// -*- C++ -*-

#ifndef QMCPLUSPLUS_MPI_COMMUNICATOR_H
#define QMCPLUSPLUS_MPI_COMMUNICATOR_H

#include <Message/OpenMP.h>

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <string>
#include <vector>
#include <utility>

#if defined(HAVE_MPI)
#include <mpi.h>
#if defined(ENABLE_SHMEM)
#include <mpp/shmem.h>
#endif
#define APP_ABORT(msg) \
  cerr << "Fatal Error. Aborting at " << msg << endl; MPI_Abort(MPI_COMM_WOLRD,0)
#else /* mpi is not found */
typedef int MPI_Comm;
typedef int MPI_Status;
typedef int MPI_Request;
typedef int MPI_Comm;
typedef int MPI_Group;
#ifndef MPI_ANY_SOURCE
#define MPI_ANY_SOURCE -1
#endif
#ifndef MPI_ANY_TAG
#define MPI_ANY_TAG -1
#endif
#define APP_ABORT(msg) \
  cerr << "Fatal Error. Aborting at " << msg << endl; abort()
#endif

#if defined(USE_BOOST_MPI)
#include <boost/mpi/environment.hpp>
#include <boost/mpi/communicator.hpp>
namespace mpi=boost::mpi;
#else
namespace mpi {

const int any_source = MPI_ANY_SOURCE;
const int any_tag = MPI_ANY_TAG;

/**@class communicator
 * @ingroup mpi
 * @brief Wrapping information on parallelism.
 *
 * This is a replacement of boost::mpi for simple applications 
 * and is not intended for real applications.
 * Very limited in functions. Currently, only single-mode or mpi-mode 
 * is available (mutually exclusive).
 */
class communicator
{

public:

  ///constructor
  communicator();

  ///copy constructor
  communicator(const MPI_Comm& comm, int oldrank);

  ///constructor with arguments
  communicator(int argc, char **argv);

  /**destructor
   * Call proper finalization of Communication library
   */
  virtual ~communicator();

  /** abort 
   * @param errcode index for the error
   */
  void abort(int errcode);

  ///return the rank of this node
  inline int rank() const {return d_mycontext;}
  ///return the size of the communicator
  inline int size() const {return d_ncontexts;}
  ///call barrier
  void barrier();
  ///operator for implicit conversion to MPI_Comm
  inline operator MPI_Comm() const
  {
    return comm_ptr;
  }
  ///return the output stream
  inline std::ostream& log()
  {
    return *my_logstream;
  }

  communicator split(int color) const;

  communicator split(int color, int key) const;

//protected:
  MPI_Comm comm_ptr;
  int d_mycontext; 
  int d_ncontexts;
  MPI_Group d_groupid;
  int d_parentid;

  std::vector<MPI_Request> requests;
  std::vector<MPI_Status> status;

  std::ostream* my_logstream;
  std::string myName;
};

///just for boost compatibility
struct environment
{
  ///static bool to call initialize once
  static bool mpi_world_invalid;
  ///static integer to count number of communicators
  static int mpi_world_copies;
  ///constructor
  environment(int argc, char **argv);
  ///static function
  static void initialize(int argc=0, char** argv=0);
  ///static function
  static void finalize();
};

}///namespace
#endif

#endif // OHMMS_COMMUNICATE_H
/***************************************************************************
 * $RCSfile$   $Author: jnkim $
 * $Revision: 2809 $   $Date: 2008-07-14 10:52:34 -0500 (Mon, 14 Jul 2008) $
 * $Id: communicator.h 2809 2008-07-14 15:52:34Z jnkim $ 
 ***************************************************************************/
